import re
import json
from datetime import datetime

def clear_text(text):
    pattern="[a-zA-Z0-9@#$%^&*(){}\-_+=:/.']+"
    return " ".join(re.findall(pattern, text))

def main():
    
#     input_file = '/home/ivy/git/Social/Social/file1/tweets.txt'
#     output_file = '/home/ivy/git/Social/Social/file1/tweets_json.txt'
#     input_file = '/home/ivy/git/Social/Social/resource/tweets_4.txt'
    for i in range(1, 15):    
        input_file = '/media/ivy/NTFS/_ubuntu_tmp/tweets/tweets_%d.txt' % i
        output_file = '/home/ivy/git/Social/Social/resource/json_tweets/json_tweets_%d.txt' % i
        clear(input_file, output_file)

def clear(input_file, output_file):    
    init_formater = '%a %b %d %H:%M:%S +0000 %Y'
    to_formater = '%Y%m%d%H%M%S'
    
    f = open(output_file, 'w')
    list = []
    count = 0
    with open(input_file) as hello_file:
        for line in hello_file:
            obj = json.loads(line)
#             print json.dumps(obj, sort_keys=True, indent=2)
            if obj['lang'] =='en':
                init_created_time = obj['created_at']
                date_time = datetime.strptime(init_created_time, init_formater).strftime(to_formater)
                status = 0
                list.append('"time": "%s"' % date_time)
                list.append('"users": "%s"' % obj['user']['id_str'])
                list.append('"followers": %d' % obj['user']['followers_count'])
                list.append('"friends": %d' % obj['user']['friends_count'])
                list.append('"text": "%s"' % clear_text(obj['text']))
                list.append('"retweet": %d' % obj['retweet_count'])
                list.append('"favorite": %d' % obj['favorite_count'])
                
                if 'quoted_status' in obj:
                    status = 1
                    quoted = obj['quoted_status']
                    list.append('"quoted_user": "%s"' % quoted['user']['id_str'])
                    list.append('"quoted_followers": %d' % quoted['user']['followers_count'])
                    list.append('"quoted_friends": %d' % quoted['user']['friends_count'])
                    list.append('"quoted_quoted_text": "%s"' % clear_text(quoted['text']))
                    list.append('"quoted_retweet": %d' % quoted['retweet_count'])
                    list.append('"quoted_favorite": %d' % quoted['favorite_count'])
                
                if 'retweeted_status' in obj:
                    status = 2
                    retweeted = obj['retweeted_status']
                    list.append('"retweeted_user": "%s"' % retweeted['user']['id_str'])
                    list.append('"retweeted_followers": %d' % retweeted['user']['followers_count'])
                    list.append('"retweeted_friends": %d' % retweeted['user']['friends_count'])
                    list.append('"retweeted_text": "%s"' % clear_text(retweeted['text']))
                    list.append('"retweeted_retweet": %d' % retweeted['retweet_count'])
                    list.append('"retweeted_favorite": %d' % retweeted['favorite_count'])
                    
                    if 'quoted_status' in obj['retweeted_status']:
                        status = 3
                        quoted = obj['retweeted_status']['quoted_status']
                        list.append('"quoted_user": "%s"' % quoted['user']['id_str'])
                        list.append('"quoted_followers": %d' % quoted['user']['followers_count'])
                        list.append('"quoted_friends": %d' % quoted['user']['friends_count'])
                        list.append('"quoted_quoted_text": "%s"' % clear_text(quoted['text']))
                        list.append('"quoted_retweet": %d' % quoted['retweet_count'])
                        list.append('"quoted_favorite": %d' % quoted['favorite_count'])
                        
                list.append('"status": %d' % status)   
                result = "{" + ", ".join(list) + "}\n"
#                 print result
                f.write(result)
                count += 1
                if count % 1000 == 0:
                    print count
            
            del list[:]
    print count
    f.flush()
    f.close()
            
    
if __name__ == '__main__':
    main() 